#!/usr/bin/env python
# encoding: utf-8
"""
@author: youfeng
@email: youfeng243@163.com
@license: Apache Licence 
@file: start_update_process.py
@time: 2016/12/13 16:18
"""
import os
import sys
import threading
import time

from common import util
from common.config_parser import ConfigParser
from common.mongo import MongDb
from logger import Gsxtlogger


class UpdateThread(threading.Thread):
    # 需要传输的表
    table_local = ['big_company_data']
    table_cmb = ['enterprise_data_cmb']

    # 更新时间
    update_time = 8

    # 需要传输的数据库
    mongo_db_from = {
        'host': '172.16.215.2',
        'port': 40042,
        'db': 'schedule_data',
        'username': 'work',
        'password': 'haizhi'
    }

    mongo_db_source = {
        'host': '172.16.215.2',
        'port': 40042,
        'db': 'company_data',
        'username': 'work',
        'password': 'haizhi'
    }

    province_zh_to_py = {
        '上海': 'shanghai',
        '云南': 'yunnan',
        '内蒙古': 'neimenggu',
        '北京': 'beijing',
        '吉林': 'jilin',
        '四川': 'sichuan',
        '天津': 'tianjin',
        '宁夏': 'ningxia',
        '安徽': 'anhui',
        '山东': 'shandong',
        '山西': 'shanxicu',
        '广东': 'guangdong',
        '广西': 'guangxi',
        '新疆': 'xinjiang',
        '江苏': 'jiangsu',
        '江西': 'jiangxi',
        '河北': 'hebei',
        '河南': 'henan',
        '浙江': 'zhejiang',
        '海南': 'hainan',
        '深圳': 'guangdong',
        '湖北': 'hubei',
        '湖南': 'hunan',
        '甘肃': 'gansu',
        '福建': 'fujian',
        '西藏': 'xizang',
        '贵州': 'guizhou',
        '辽宁': 'liaoning',
        '重庆': 'chongqing',
        '陕西': 'shanxi',
        '青海': 'qinghai',
        '黑龙江': 'heilongjiang',
    }

    province_py_to_zh = {
        'shanghai': ['上海'],
        'yunnan': ['云南'],
        'neimenggu': ['内蒙古'],
        'beijing': ['北京'],
        'jilin': ['吉林'],
        'sichuan': ['四川'],
        'tianjin': ['天津'],
        'ningxia': ['宁夏'],
        'anhui': ['安徽'],
        'shandong': ['山东'],
        'shanxicu': ['山西'],
        'guangdong': ['广东', '深圳'],
        'guangxi': ['广西'],
        'xinjiang': ['新疆'],
        'jiangsu': ['江苏'],
        'jiangxi': ['江西'],
        'hebei': ['河北'],
        'henan': ['河南'],
        'zhejiang': ['浙江'],
        'hainan': ['海南'],
        'hubei': ['湖北'],
        'hunan': ['湖南'],
        'gansu': ['甘肃'],
        'fujian': ['福建'],
        'xizang': ['西藏'],
        'guizhou': ['贵州'],
        'liaoning': ['辽宁'],
        'chongqing': ['重庆'],
        'shanxi': ['陕西'],
        'qinghai': ['青海'],
        'heilongjiang': ['黑龙江'],
    }

    # 需要读取的属性字段
    field_list = ['_id', 'province']

    # 内置本地配置文件路径
    file_path = 'config/update_file.conf'

    def __init__(self, config_file='config/cmb_gsxt_search.conf'):
        threading.Thread.__init__(self)

        # 初始化配置路径信息
        self.base_path = os.path.dirname(os.path.realpath(__file__))
        self.config_file = self.base_path + '/' + config_file
        self.update_file = self.base_path + '/' + self.file_path

        # 创建日志
        self.log = Gsxtlogger('start_update_process').get_logger()

    @staticmethod
    def get_config_list(config_file):
        # 获得所有列表信息
        conf_parse = ConfigParser(config_file)
        config_list = conf_parse.get_all_session()
        return config_list

    def get_table_info(self, config_file):
        table_list = {}
        config_list = self.get_config_list(config_file)
        for key, value in config_list.iteritems():
            table_list[key] = value.get('source_table', '')
            self.log.info('key = {key} table = {table}'.format(key=key, table=table_list[key]))

        return table_list

    def __get_field_list(self):
        return self.field_list

    def __load_config_info(self):
        config_list = self.get_config_list(self.update_file)

        # 输出所有配置信息
        self.log.info(config_list)

        update_info = config_list.get('update_info', None)
        if update_info is None:
            return

        mongo_db_from = eval(update_info.get('mongo_db_from', 'None'))
        if mongo_db_from is not None:
            self.mongo_db_from = mongo_db_from

        mongo_db_source = eval(update_info.get('mongo_db_source', 'None'))
        if mongo_db_source is not None:
            self.mongo_db_source = mongo_db_source

        table_local = eval(update_info.get('table_local', 'None'))
        if table_local is not None:
            self.table_local = table_local

        table_cmb = eval(update_info.get('table_cmb', 'None'))
        if table_cmb is not None:
            self.table_cmb = table_cmb

        update_time = int(update_info.get('update_time', '0'))
        if update_time != 0:
            self.update_time = update_time

    # 数据传输
    def __data_transmissions(self):

        # 加载配置文件信息
        self.__load_config_info()

        # 连接mongodb
        source_db = MongDb(self.mongo_db_source['host'], self.mongo_db_source['port'], self.mongo_db_source['db'],
                           self.mongo_db_source['username'], self.mongo_db_source['password'], log=self.log)

        from_db = MongDb(self.mongo_db_from['host'], self.mongo_db_from['port'], self.mongo_db_from['db'],
                         self.mongo_db_from['username'], self.mongo_db_from['password'], log=self.log)

        table_list = self.get_table_info(self.config_file)
        for key, table in table_list.iteritems():
            # 创建索引
            source_db.create_index(table, [("province", MongDb.ASCENDING)])
            source_db.create_index(table, [("company_name", MongDb.ASCENDING)])

        # 导入本地数据
        for table in self.table_local:
            self.__data_transmission_local(source_db, from_db, table, table_list)

        # 导入招行数据
        for table in self.table_cmb:
            self.__data_transmission_cmb(source_db, from_db, table, table_list)

    # 加载到内存模式
    def __get_all_company_list(self, data_base, table, field_list, where=None):
        company_list = []
        start_time = time.time()
        self.log.info('开始加载种子列表..table = {table}'.format(table=table))
        for item in data_base.select_field(table, where, field_list):
            company_list.append(item)
        self.log.info('加载数据库数据个数为: size = {size}'.format(size=len(company_list)))
        end_time = time.time()

        self.log.info('加载种子列表起始时间: {st}'.format(st=start_time))
        self.log.info('加载种子列表结束时间: {et}'.format(et=end_time))
        self.log.info('加载种子列表消耗时间: {t}s'.format(t=end_time - start_time))

        return company_list

    # 遍历模式
    @staticmethod
    def __get_iterator_company_list(data_base, table, field_list, where=None):
        return data_base.traverse_field(table, where, field_list)

    def __get_source_list(self, data_base, table, field_list, where=None):
        total_seed = data_base.select_count(table)
        self.log.info('当前种子表: {table} 数目为 {size}'.format(
            table=table, size=total_seed))

        if total_seed <= 4000000:
            company_list = self.__get_all_company_list(data_base, table, field_list, where)
            self.log.info('加载完成 种子列表...')
        else:
            company_list = self.__get_iterator_company_list(data_base, table, field_list, where)
            self.log.info('迭代遍历 种子列表...')

        return company_list

    # 这里是读取招行数据的逻辑
    def __data_transmission_cmb(self, source_db, from_db, source_table, table_list):

        self.log.info('开始更新招行数据库种子列表..')
        # 计时
        start_time = time.time()

        # 统计
        total_load = 0
        select_param = {}

        # 读取数据
        company_list = self.__get_source_list(from_db, source_table, self.__get_field_list(),
                                              select_param)
        # 传输数据
        for item in company_list:
            _id = item.get('_id', None)
            province = item.get('province', None)
            if _id is None or province is None:
                continue

            if province not in table_list:
                continue

            data = {'_id': _id, 'province': province, 'company_name': _id, 'from': source_table,
                    'in_time': util.get_now_time()}
            if source_db.find_one(table_list[province], {'_id': _id}) is None:
                source_db.insert(table_list[province], data)
                total_load += 1

        # 清除使用内存
        if isinstance(company_list, list):
            del company_list
        end_time = time.time()

        self.log.info('更新招行数据起始时间: {st}'.format(st=start_time))
        self.log.info('更新招行数据结束时间: {et}'.format(et=end_time))
        self.log.info('更新招行数据消耗时间: {t}s'.format(t=end_time - start_time))
        self.log.info('总共更新招行数据为: {total}'.format(total=total_load))

    def __data_transmission_local(self, source_db, from_db, source_table, table_list):

        self.log.info('开始更新本地数据库种子列表..')
        # 计时
        start_time = time.time()

        # 统计
        total_load = 0
        select_param = {}

        # 读取公司数据
        company_list = self.__get_source_list(from_db, source_table, self.__get_field_list(), select_param)

        # 传输数据
        for item in company_list:
            _id = item.get('_id', None)
            province = item.get('province', None)
            if _id is None or province is None:
                continue
            _id = _id.strip().strip('"').strip('\'')
            if _id == '':
                continue
            province = province.strip().encode('utf-8')
            if province == '':
                continue
            if province not in self.province_zh_to_py:
                continue

            if self.province_zh_to_py[province] not in table_list:
                continue

            data = {'_id': _id, 'province': self.province_zh_to_py[province], 'company_name': _id, 'from': source_table,
                    'in_time': util.get_now_time()}
            if source_db.find_one(table_list[self.province_zh_to_py[province]], {'_id': _id}) is None:
                source_db.insert(table_list[self.province_zh_to_py[province]], data)
                total_load += 1

                # if source_db.find_one('cmb_important_all', {'_id': _id}) is None:
                #     source_db.insert('cmb_important_all', data)
                #     total_load += 1
                #     self.log.info('index = {index}'.format(index=total_load))

        # 清除使用内存
        if isinstance(company_list, list):
            del company_list[:]

        end_time = time.time()

        self.log.info('更新本地数据起始时间: {st}'.format(st=start_time))
        self.log.info('更新本地数据结束时间: {et}'.format(et=end_time))
        self.log.info('更新本地数据消耗时间: {t}s'.format(t=end_time - start_time))
        self.log.info('总共更新本地数据为: {total}'.format(total=total_load))

    def run(self):
        while True:
            self.__data_transmissions()
            self.log.info('更新完成..进入休眠..为期{timer}小时..'.format(timer=self.update_time))
            time.sleep(3600 * self.update_time)


def main():
    config = 'config/cmb_gsxt_searchlist.conf'

    length = len(sys.argv)
    if length > 1:
        config = sys.argv[1]
    update_thread = UpdateThread(config)
    update_thread.start()


if __name__ == '__main__':
    main()
